{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"mnist\" + os.sep\n",
    "\n",
    "DENSE_TRAIN_FILE = \"dense_train.ak\";\n",
    "DENSE_TEST_FILE = \"dense_test.ak\";\n",
    "SPARSE_TRAIN_FILE = \"sparse_train.ak\";\n",
    "SPARSE_TEST_FILE = \"sparse_test.ak\";\n",
    "PCA_MODEL_FILE = \"pca_model.ak\";\n",
    "\n",
    "VECTOR_COL_NAME = \"vec\";\n",
    "LABEL_COL_NAME = \"label\";\n",
    "PREDICTION_COL_NAME = \"pred\";\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1\n",
    "df = pd.DataFrame(\n",
    "    [\n",
    "        [\"ALABAMA\", 14.2, 25.2, 96.8, 278.3, 1135.5, 1881.9, 280.7],\n",
    "        [\"ALASKA\", 10.8, 51.6, 96.8, 284.0, 1331.7, 3369.8, 753.3],\n",
    "        [\"ARIZONA\", 9.5, 34.2, 138.2, 312.3, 2346.1, 4467.4, 439.5],\n",
    "        [\"ARKANSAS\", 8.8, 27.6, 83.2, 203.4, 972.6, 1862.1, 183.4],\n",
    "        [\"CALIFORNIA\", 11.5, 49.4, 287.0, 358.0, 2139.4, 3499.8, 663.5],\n",
    "        [\"COLORADO\", 6.3, 42.0, 170.7, 292.9, 1935.2, 3903.2, 477.1],\n",
    "        [\"CONNECTICUT\", 4.2, 16.8, 129.5, 131.8, 1346.0, 2620.7, 593.2],\n",
    "        [\"DELAWARE\", 6.0, 24.9, 157.0, 194.2, 1682.6, 3678.4, 467.0],\n",
    "        [\"FLORIDA\", 10.2, 39.6, 187.9, 449.1, 1859.9, 3840.5, 351.4],\n",
    "        [\"GEORGIA\", 11.7, 31.1, 140.5, 256.5, 1351.1, 2170.2, 297.9],\n",
    "        [\"HAWAII\", 7.2, 25.5, 128.0, 64.1, 1911.5, 3920.4, 489.4],\n",
    "        [\"IDAHO\", 5.5, 19.4, 39.6, 172.5, 1050.8, 2599.6, 237.6],\n",
    "        [\"ILLINOIS\", 9.9, 21.8, 211.3, 209.0, 1085.0, 2828.5, 528.6],\n",
    "        [\"INDIANA\", 7.4, 26.5, 123.2, 153.5, 1086.2, 2498.7, 377.4],\n",
    "        [\"IOWA\", 2.3, 10.6, 41.2, 89.8, 812.5, 2685.1, 219.9],\n",
    "        [\"KANSAS\", 6.6, 22.0, 100.7, 180.5, 1270.4, 2739.3, 244.3],\n",
    "        [\"KENTUCKY\", 10.1, 19.1, 81.1, 123.3, 872.2, 1662.1, 245.4],\n",
    "        [\"LOUISIANA\", 15.5, 30.9, 142.9, 335.5, 1165.5, 2469.9, 337.7],\n",
    "        [\"MAINE\", 2.4, 13.5, 38.7, 170.0, 1253.1, 2350.7, 246.9],\n",
    "        [\"MARYLAND\", 8.0, 34.8, 292.1, 358.9, 1400.0, 3177.7, 428.5],\n",
    "        [\"MASSACHUSETTS\", 3.1, 20.8, 169.1, 231.6, 1532.2, 2311.3, 1140.1],\n",
    "        [\"MICHIGAN\", 9.3, 38.9, 261.9, 274.6, 1522.7, 3159.0, 545.5],\n",
    "        [\"MINNESOTA\", 2.7, 19.5, 85.9, 85.8, 1134.7, 2559.3, 343.1],\n",
    "        [\"MISSISSIPPI\", 14.3, 19.6, 65.7, 189.1, 915.6, 1239.9, 144.4],\n",
    "        [\"MISSOURI\", 9.6, 28.3, 189.0, 233.5, 1318.3, 2424.2, 378.4],\n",
    "        [\"MONTANA\", 5.4, 16.7, 39.2, 156.8, 804.9, 2773.2, 309.2],\n",
    "        [\"NEBRASKA\", 3.9, 18.1, 64.7, 112.7, 760.0, 2316.1, 249.1],\n",
    "        [\"NEVADA\", 15.8, 49.1, 323.1, 355.0, 2453.1, 4212.6, 559.2],\n",
    "        [\"NEW HAMPSHIRE\", 3.2, 10.7, 23.2, 76.0, 1041.7, 2343.9, 293.4],\n",
    "        [\"NEW JERSEY\", 5.6, 21.0, 180.4, 185.1, 1435.8, 2774.5, 511.5],\n",
    "        [\"NEW MEXICO\", 8.8, 39.1, 109.6, 343.4, 1418.7, 3008.6, 259.5],\n",
    "        [\"NEW YORK\", 10.7, 29.4, 472.6, 319.1, 1728.0, 2782.0, 745.8],\n",
    "        [\"NORTH CAROLINA\", 10.6, 17.0, 61.3, 318.3, 1154.1, 2037.8, 192.1],\n",
    "        [\"NORTH DAKOTA\", 0.9, 9.0, 13.3, 43.8, 446.1, 1843.0, 144.7],\n",
    "        [\"OHIO\", 7.8, 27.3, 190.5, 181.1, 1216.0, 2696.8, 400.4],\n",
    "        [\"OKLAHOMA\", 8.6, 29.2, 73.8, 205.0, 1288.2, 2228.1, 326.8],\n",
    "        [\"OREGON\", 4.9, 39.9, 124.1, 286.9, 1636.4, 3506.1, 388.9],\n",
    "        [\"PENNSYLVANIA\", 5.6, 19.0, 130.3, 128.0, 877.5, 1624.1, 333.2],\n",
    "        [\"RHODE ISLAND\", 3.6, 10.5, 86.5, 201.0, 1489.5, 2844.1, 791.4],\n",
    "        [\"SOUTH CAROLINA\", 11.9, 33.0, 105.9, 485.3, 1613.6, 2342.4, 245.1],\n",
    "        [\"SOUTH DAKOTA\", 2.0, 13.5, 17.9, 155.7, 570.5, 1704.4, 147.5],\n",
    "        [\"TENNESSEE\", 10.1, 29.7, 145.8, 203.9, 1259.7, 1776.5, 314.0],\n",
    "        [\"TEXAS\", 13.3, 33.8, 152.4, 208.2, 1603.1, 2988.7, 397.6],\n",
    "        [\"UTAH\", 3.5, 20.3, 68.8, 147.3, 1171.6, 3004.6, 334.5],\n",
    "        [\"VERMONT\", 1.4, 15.9, 30.8, 101.2, 1348.2, 2201.0, 265.2],\n",
    "        [\"VIRGINIA\", 9.0, 23.3, 92.1, 165.7, 986.2, 2521.2, 226.7],\n",
    "        [\"WASHINGTON\", 4.3, 39.6, 106.2, 224.8, 1605.6, 3386.9, 360.3],\n",
    "        [\"WEST VIRGINIA\", 6.0, 13.2, 42.2, 90.9, 597.4, 1341.7, 163.3],\n",
    "        [\"WISCONSIN\", 2.8, 12.9, 52.2, 63.7, 846.9, 2614.2, 220.7],\n",
    "        [\"WYOMING\", 5.4, 21.9, 39.7, 173.9, 811.6, 2772.2, 282.0]\n",
    "    ]\n",
    ")\n",
    "\n",
    "schema_str = \"state string, murder double, rape double, robbery double, \"\\\n",
    "    + \"assault double, burglary double, larceny double, auto double\"\n",
    "source = BatchOperator.fromDataframe(df, schema_str)\n",
    "\n",
    "source.lazyPrint(10, \"Origin data\");\n",
    "\n",
    "pca_result = PCA()\\\n",
    "    .setK(4)\\\n",
    "    .setSelectedCols([\"murder\", \"rape\", \"robbery\", \"assault\", \n",
    "                      \"burglary\", \"larceny\", \"auto\"])\\\n",
    "    .setPredictionCol(VECTOR_COL_NAME)\\\n",
    "    .enableLazyPrintModelInfo()\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        VectorToColumnsBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setSchemaStr(\"prin1 double, prin2 double, prin3 double, prin4 double\")\\\n",
    "            .setReservedCols([\"state\"])\n",
    "    )\\\n",
    "    .lazyPrint(10, \"state with principle components\");\n",
    "\n",
    "pca_result\\\n",
    "    .select(\"state, prin1\")\\\n",
    "    .orderBy(\"prin1\", limit = 100, order = 'desc')\\\n",
    "    .lazyPrint(-1, \"Order by prin1\");\n",
    "\n",
    "pca_result\\\n",
    "    .select(\"state, prin2\")\\\n",
    "    .orderBy(\"prin2\", limit = 100, order = 'desc')\\\n",
    "    .lazyPrint(-1, \"Order by prin2\");\n",
    "\n",
    "BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_2\n",
    "std_pca = Pipeline()\\\n",
    "    .add(\n",
    "        StandardScaler()\\\n",
    "            .setSelectedCols([\"murder\", \"rape\", \"robbery\", \"assault\", \n",
    "                              \"burglary\", \"larceny\", \"auto\"])\n",
    "    )\\\n",
    "    .add(\n",
    "        PCA()\\\n",
    "            .setCalculationType('COV')\\\n",
    "            .setK(4)\\\n",
    "            .setSelectedCols([\"murder\", \"rape\", \"robbery\", \"assault\", \n",
    "                              \"burglary\", \"larceny\", \"auto\"])\\\n",
    "            .setPredictionCol(VECTOR_COL_NAME)\\\n",
    "            .enableLazyPrintModelInfo()\n",
    "    );\n",
    "\n",
    "std_pca\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        VectorToColumnsBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setSchemaStr(\"prin1 double, prin2 double, \" \n",
    "                          + \"prin3 double, prin4 double\")\\\n",
    "            .setReservedCols([\"state\"])\n",
    "    )\\\n",
    "    .lazyPrint(10, \"state with principle components\");\n",
    "BatchOperator.execute();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3\n",
    "\n",
    "source = AkSourceBatchOp().setFilePath(DATA_DIR + SPARSE_TRAIN_FILE);\n",
    "\n",
    "source\\\n",
    "    .link(\n",
    "        PcaTrainBatchOp()\\\n",
    "            .setK(39)\\\n",
    "            .setCalculationType('COV')\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .lazyPrintModelInfo()\n",
    "    )\\\n",
    "    .link(\n",
    "        AkSinkBatchOp()\\\n",
    "            .setFilePath(DATA_DIR + PCA_MODEL_FILE)\\\n",
    "            .setOverwriteSink(True)\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "\n",
    "\n",
    "sw = Stopwatch();\n",
    "\n",
    "kmeans = KMeans()\\\n",
    "    .setK(10)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME);\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "kmeans\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KMeans\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "pca_result = PcaPredictBatchOp()\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setPredictionCol(VECTOR_COL_NAME)\\\n",
    "    .linkFrom(\n",
    "        AkSourceBatchOp().setFilePath(DATA_DIR + PCA_MODEL_FILE),\n",
    "        source\n",
    "    );\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "kmeans\\\n",
    "    .fit(pca_result)\\\n",
    "    .transform(pca_result)\\\n",
    "    .link(\n",
    "        EvalClusterBatchOp()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KMeans + PCA\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4\n",
    "\n",
    "useLocalEnv(4)\n",
    "\n",
    "dense_train_data = AkSourceBatchOp().setFilePath(DATA_DIR + DENSE_TRAIN_FILE);\n",
    "dense_test_data = AkSourceBatchOp().setFilePath(DATA_DIR + DENSE_TEST_FILE);\n",
    "sparse_train_data = AkSourceBatchOp().setFilePath(DATA_DIR + SPARSE_TRAIN_FILE);\n",
    "sparse_test_data = AkSourceBatchOp().setFilePath(DATA_DIR + SPARSE_TEST_FILE);\n",
    "\n",
    "sw = Stopwatch();\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "KnnClassifier()\\\n",
    "    .setK(3)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .fit(dense_train_data)\\\n",
    "    .transform(dense_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KnnClassifier Dense\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "KnnClassifier()\\\n",
    "    .setK(3)\\\n",
    "    .setVectorCol(VECTOR_COL_NAME)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .fit(sparse_train_data)\\\n",
    "    .transform(sparse_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"KnnClassifier Sparse\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "Pipeline()\\\n",
    "    .add(\n",
    "        PCA()\\\n",
    "            .setK(39)\\\n",
    "            .setCalculationType('COV')\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(VECTOR_COL_NAME)\n",
    "    )\\\n",
    "    .add(\n",
    "        KnnClassifier()\\\n",
    "            .setK(3)\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\n",
    "    )\\\n",
    "    .fit(dense_train_data)\\\n",
    "    .transform(dense_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Knn with PCA Dense\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "Pipeline()\\\n",
    "    .add(\n",
    "        PCA()\\\n",
    "            .setK(39)\\\n",
    "            .setCalculationType('COV')\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(VECTOR_COL_NAME)\n",
    "    )\\\n",
    "    .add(\n",
    "        KnnClassifier()\\\n",
    "            .setK(3)\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\n",
    "    )\\\n",
    "    .fit(sparse_train_data)\\\n",
    "    .transform(sparse_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Knn with PCA Sparse\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "Pipeline()\\\n",
    "    .add(\n",
    "        PCAModel()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(VECTOR_COL_NAME)\\\n",
    "            .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + PCA_MODEL_FILE))\n",
    "    )\\\n",
    "    .add(\n",
    "        KnnClassifier()\\\n",
    "            .setK(3)\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\n",
    "    )\\\n",
    "    .fit(dense_train_data)\\\n",
    "    .transform(dense_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Knn PCAModel Dense\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());\n",
    "\n",
    "sw.reset();\n",
    "sw.start();\n",
    "Pipeline()\\\n",
    "    .add(\n",
    "        PCAModel()\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setPredictionCol(VECTOR_COL_NAME)\\\n",
    "            .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + PCA_MODEL_FILE))\n",
    "    )\\\n",
    "    .add(\n",
    "        KnnClassifier()\\\n",
    "            .setK(3)\\\n",
    "            .setVectorCol(VECTOR_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\n",
    "    )\\\n",
    "    .fit(sparse_train_data)\\\n",
    "    .transform(sparse_test_data)\\\n",
    "    .link(\n",
    "        EvalMultiClassBatchOp()\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .lazyPrintMetrics(\"Knn PCAModel Sparse\")\n",
    "    );\n",
    "BatchOperator.execute();\n",
    "sw.stop();\n",
    "print(sw.getElapsedTimeSpan());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
